#include "Dictionary/DictProducer.h"


DictProducer::DictProducer()
:_jieba("data/dict/jieba.dict.utf8", 
    "data/dict/hmm_model.utf8", 
    "data/dict/user.dict.utf8",
    "data/dict/idf.utf8",
    "data/dict/stop_words.utf8")
{   
    //加载英文停止词
    ifstream ifs_stop("data/yuliao/stop_words_eng.txt");
    if(!ifs_stop.is_open()){
        LogError("stop_words_eng.txt open failed");
    }
    string word;
    while(ifs_stop>>word){
        _stop_en.insert(word);
    }
    //加载中文停止词
    ifstream ifs_ch_stop("data/yuliao/stop_words_zh.txt");
    if(!ifs_ch_stop.is_open()){
        LogError("stop_words_eng.txt open failed");
    }
    string ch_word;
    while(ifs_ch_stop>>ch_word){
        _stop_ch.insert(ch_word);
    }
}
//生成英文词典
void DictProducer::buildEnDict(const string & filename)
{
    ifstream ifs(filename);//加载文件
    if(!ifs.is_open()){
        LogError("%s open failed",filename);
    }

    ostringstream oss;
    oss<<ifs.rdbuf();//将文件输入到字符串输入流中
    string data = oss.str();
     
    //处理非法字符
    for(auto & ch : data)
    {
        if(!isalpha(ch))
        {
            ch = ' ';
        }else if(isupper(ch))//将大写转成小写
        {
            ch = tolower(ch);
        }
    }
    
    //切割成单词
    istringstream iss(data);
    string word;
    map<string, int> _dict;
    while(iss>>word){//将文件切成单词
        if(_stop_en.find(word) == _stop_en.end()){//检测停止词
            ++_dict[word];  //放入临时容器map中
        }
    }

    ofstream ofs("data/dict_en.dat");
    for(auto & word : _dict){
        //ofs << word.first << " "<<word.second<<endl;//输出到文件
        ofs<<word<<"\n";
        pair<string,int> tep(word.first,word.second);
        _dict_vector.push_back(tep);         //保存到vector容器
    }

    ifs.close();
    ofs.close();
}

//生成中文词典
void DictProducer::buildChDict()
{
    //打开中文文件夹
    DIR * dir = opendir("data/yuliao/art");
    if(dir == NULL){
        LogError("open data/yuliao failed");
    }
    //将文件夹中的所有文件内容读取到vector<string>中
    struct dirent * ptr;
    ostringstream oss;
    while((ptr = readdir(dir)) != NULL){  //循环处理文件夹中的每个文件
        string filename("data/yuliao/art/"+ string(ptr->d_name));
        ifstream ifs(filename);
        oss<<ifs.rdbuf();//将文件输入到字符串输入流中
    }
        //将所有文件内容放入一个string
        string data = oss.str();
        //对数据进行预处理，删除换行符
        for(auto ch : data){
            if( ch == '\n'){
                ch = ' ';
            }
        }
        //使用结巴分词
        vector<string> words;
        _jieba.Cut(data,words,true);
        //存入临时容器map，统计词频
        map<string, int> dict;
        for(auto word : words){
            if(_stop_ch.find(word) == _stop_ch.end()){//与停止词比较
                ++dict[word];
            }
        }

        //输出到文件，并存入vector
        ofstream ofs_ch("data/dict_ch.dat");
        for(auto word : dict){
            ofs_ch << word<<"\n";
            pair<string,int> tep(word.first,word.second);
            _dict_vector.push_back(tep);         //保存到vector容器
        }
    

    //输出中英文词典
    ofstream ofs_dirc("data/dict.dat");
    for(auto word : _dict_vector){
        ofs_dirc << word<<"\n";
    }

    closedir(dir);
}
//创建索引
void DictProducer::createIndex()
{   
    //将词分成字母，和单个汉字
    for(size_t i=0;i<_dict_vector.size();++i){//遍历每个单词
        //从vector中取出每个单词
        string word = _dict_vector[i].first;   //map<string,set<int>>
        //遍历单词的每个字母
        for(size_t j = 0 ;j <word.size();)
        {           // 1000 0000 & ch

            if((word[j] & 0x80) == 0){
                // string key = word.substr(j,1);
                // string val = _dict_vector[i].first;
                // LogDebug("%s %s",key.c_str(),val.c_str());
                //放入容器
                _word_index[word.substr(j,1)].insert(i);
               // cout << "english ch = " << word.substr(j,1) << "\n";
                ++j;
            } else if((word[j] & 0xE0) == 0xC0){
                _word_index[word.substr(j,2)].insert(i);
               //cout << "english ch = " << word.substr(j,2) << "\n";
                j += 2;
            }else if((word[j] & 0xF0) == 0xE0){
                _word_index[word.substr(j,3)].insert(i);
                //cout << "english ch = " << word.substr(j,3) << "\n";
                j += 3;
            }else if((word[j] & 0xF8) == 0xF0){
                _word_index[word.substr(j,4)].insert(i);
                //cout << "english ch = " << word.substr(j,4) << "\n";
                j += 4;
            }else{
                ++j; //避免死循环
            }
        }
    }

    ofstream ofst("data/word_index.dat");
    for(auto & word : _word_index){
        //ofst << word.first << " ";//保存到文件
        // for(auto & a : word.second){
        //     //ofst<<a<<" ";
        //     cout<<a<<" ";
        // }
        //ofst<<endl;
        ofst<<word<<"\n";
        
    }
    ofst.close();
}


void DictProducer::print_dict()
{
    for(auto it : _dict_vector){
        cout<< it.first << " "<<it.second<<endl;
    }
}

void DictProducer::print_stop_en(){
    for(auto it : _stop_en){
        cout<< it <<endl;
    }
}

void DictProducer::Surch(const string &ch){
    string a = ch;
    LogDebug(a.c_str());
    for(auto it : _word_index[a]){//遍历字符a的set集合
        //通过下标找到所有相关的单词
        cout<<_dict_vector[it]<<"\n";
    }
}

